Data exploration of House Price Predictions¶

The data is downloaded from Kaggle and contains data about houses and around ~80 variables. The goal is to use these explanatory variables to predict the House Prices. Here, we are dealing with a regression problem. As we want to conduct a classification, as well, we will also divide the SalePrice in three categories, namely "low", "middle" and "upper class". We will then attempt to predict these 3 classes.

https://www.kaggle.com/c/house-prices-advanced-regression-techniques

This notebook is to explore the data, to understand the basic relationships between the variables and to get a feeling about which variables might be good predictors for the House prices. There will be a separate notebook containing statistical and machine learning models for the predictions.

Author: Julia Hammerer, Vanessa Mai Last Changes: 18.11.2018

# load packages
%matplotlib inline
import numpy as np
import pandas as pd
import seaborn as sns
import pandas_profiling
import missingno as msno
import matplotlib.pyplot as plt
import plotly.graph_objs as go
import warnings
import math

from pandas.tools.plotting import table
from plotly.offline import init_notebook_mode
from plotly.offline import iplot
from plotly.offline import plot
from scipy.stats import mannwhitneyu
from statsmodels.distributions.empirical_distribution import ECDF
from scipy import stats
from scipy.stats import pearsonr
from scipy.stats import norm

from helper import na_ratio_table
from helper import corr_heatmap
from helper import corr_matrix_1

C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\pandas_profiling\plot.py:15: UserWarning: 
This call to matplotlib.use() has no effect because the backend has already
been chosen; matplotlib.use() must be called *before* pylab, matplotlib.pyplot,
or matplotlib.backends is imported for the first time.

The backend was *originally* set to 'module://ipykernel.pylab.backend_inline' by the following code:
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\ipykernel\__main__.py", line 3, in <module>
    app.launch_new_instance()
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\traitlets\config\application.py", line 658, in launch_instance
    app.start()
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 505, in start
    self.io_loop.start()
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\tornado\platform\asyncio.py", line 132, in start
    self.asyncio_loop.run_forever()
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\asyncio\base_events.py", line 422, in run_forever
    self._run_once()
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\asyncio\base_events.py", line 1434, in _run_once
    handle._run()
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\asyncio\events.py", line 145, in _run
    self._callback(*self._args)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\tornado\ioloop.py", line 758, in _run_callback
    ret = callback()
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\tornado\stack_context.py", line 300, in null_wrapper
    return fn(*args, **kwargs)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\tornado\gen.py", line 1233, in inner
    self.run()
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\tornado\gen.py", line 1147, in run
    yielded = self.gen.send(value)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 370, in dispatch_queue
    yield self.process_one()
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\tornado\gen.py", line 346, in wrapper
    runner = Runner(result, future, yielded)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\tornado\gen.py", line 1080, in __init__
    self.run()
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\tornado\gen.py", line 1147, in run
    yielded = self.gen.send(value)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 357, in process_one
    yield gen.maybe_future(dispatch(*args))
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\tornado\gen.py", line 326, in wrapper
    yielded = next(result)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 267, in dispatch_shell
    yield gen.maybe_future(handler(stream, idents, msg))
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\tornado\gen.py", line 326, in wrapper
    yielded = next(result)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 534, in execute_request
    user_expressions, allow_stdin,
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\tornado\gen.py", line 326, in wrapper
    yielded = next(result)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 294, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 536, in run_cell
    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2817, in run_cell
    raw_cell, store_history, silent, shell_futures)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2843, in _run_cell
    return runner(coro)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\IPython\core\async_helpers.py", line 67, in _pseudo_sync_runner
    coro.send(None)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3018, in run_cell_async
    interactivity=interactivity, compiler=compiler, result=result)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3183, in run_ast_nodes
    if (yield from self.run_code(code, result)):
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3265, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-2-bafafaff196f>", line 2, in <module>
    get_ipython().run_line_magic('matplotlib', 'inline')
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2285, in run_line_magic
    result = fn(*args,**kwargs)
  File "<decorator-gen-108>", line 2, in matplotlib
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\IPython\core\magic.py", line 187, in <lambda>
    call = lambda f, *a, **k: f(*a, **k)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\IPython\core\magics\pylab.py", line 99, in matplotlib
    gui, backend = self.shell.enable_matplotlib(args.gui)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3353, in enable_matplotlib
    pt.activate_matplotlib(backend)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\IPython\core\pylabtools.py", line 314, in activate_matplotlib
    matplotlib.pyplot.switch_backend(backend)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\matplotlib\pyplot.py", line 231, in switch_backend
    matplotlib.use(newbackend, warn=False, force=True)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\matplotlib\__init__.py", line 1410, in use
    reload(sys.modules['matplotlib.backends'])
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\importlib\__init__.py", line 166, in reload
    _bootstrap._exec(spec, module)
  File "C:\Users\maiv2\AppData\Local\Continuum\anaconda3\lib\site-packages\matplotlib\backends\__init__.py", line 16, in <module>
    line for line in traceback.format_stack()


  matplotlib.use(BACKEND)

warnings.filterwarnings('ignore')

#load data
# we have two files, since this is a part of a kaggle competition,
# only the training-set contains the target variable
# we will use that for the whole analysis

df=pd.read_csv("../data/house_prices_train.csv")

print("Number of records and variables: ",df.shape)

Number of records and variables:  (1460, 81)

Data Profile¶

# for a first overview, we apply the pandas-profile report
# it provides simple histograms, distributions, missingness 
# and correlations for all variables

pandas_profiling.ProfileReport(df)

A description of all data fields can be found on the Kaggle site: https://www.kaggle.com/c/house-prices-advanced-regression-techniques/data. Most of them are self-explanatory though.

Around half the variables are categorical and the other half are numerical. For the categorical variables there will be need to use hot-one-encoding for incorporating them into the prediction models.

We can already spot some correlations that look promising. Some of them also are also expected and won't give us further insights. We are particularly interested in correlations with our target variable

OverallQual - SalePrice
GrLivArea - SalePrice
FullBath - SalePrice
GarageYrBlt - YearRemodAdd
LotFrontage - lotArea
TotRmsAbvGrd - GrLivArea
BsmtUnfSF - BsmtFinSF1: negative correlation

Also, we can detect variables that probably won't be of much use e.g.

Street: only two values, of which one is extremly low.
Utilities: Almost constant with two values, of which the other one has only one record

df.select_dtypes(include=['object']).head()

Missing values¶

Let's check the missingness in more detail

display(na_ratio_table(df)[na_ratio_table(df)["NA_COUNT"]>0])
display(na_ratio_table(df)[na_ratio_table(df)["NA_COUNT"]>0].shape)

(19, 2)

We have 19 variables that contain missing values. Most of them mean that the feature is simply not available for that property. However for a few, this can indicate a data quality issue:

Electrical: the type is not stated, it is improbable that there is no electrical system at all.
LotFrontage: a building should always have a lotfrontage

As for "Electrical" only one record is missing, we can simply filter this out, or even ignore this. For the LotFrontage we can apply some imputation-techniques if necessary.

# we test if the data is randomly missing, or if there are some patterns in the missingness
# this helps us indicate whether there are data quality issues or if the missingness is part of the data
msno.heatmap(df)

<matplotlib.axes._subplots.AxesSubplot at 0x1a34fde3438>

As expected, we can see that some of the variables are always missing together, which makes absolutely sense. Example: All Garage related variables are always missing together. Reason: no garage -> no values for any garage features. The other group of variables missing together is related to the basement. Because of these correlations, we might run into some multicollinearity issues in the modelling part later on. "Multicollinearity is a state of very high intercorrelations or inter-associations among the independent variables. It is therefore a type of disturbance in the data, and if present in the data the statistical inferences made about the data may not be reliable." (https://www.statisticssolutions.com/multicollinearity/, Accesed on: 22.11.2018)

Sanity checks¶

We're going to check if there are some inconsistencies in the data or duplicates, etc. (Quality assessment)

# any duplicates?
df[df.duplicated(keep=False)]

# any built year before sold year?
df.query('YearBuilt > YrSold')

Cleanse data¶

# remove useless columns
df=df.drop(columns=["Id", "Street", "Utilities"])

# remove the missing record for Electrical
df=df[df["Electrical"].isna()==False]

Explorations¶

df.describe()

Let's once again check the distribution of the SalePrice

sns.distplot(df["SalePrice"], hist=False, label="SalePrice", fit=norm,  kde_kws={"shade": True})

<matplotlib.axes._subplots.AxesSubplot at 0x1a350680d68>

The distribution is slightly skewed.

stats.probplot(df['SalePrice'], plot=plt)

((array([-3.30494753, -3.04772643, -2.90468264, ...,  2.90468264,
          3.04772643,  3.30494753]),
  array([ 34900,  35311,  37900, ..., 625000, 745000, 755000], dtype=int64)),
 (74187.860388132, 180930.39479095273, 0.9320032714937404))

SalePrice does not have a normal distribution.

Correlation Analysis for numeric features¶

# we check for further correlations using different plots
NUM_FEATURES =df.select_dtypes(include=[np.number]).columns.tolist()

df_num=df[NUM_FEATURES]

df_corr=df_num.corr()

corr_heatmap(df_corr, figsize=(20, 16))

corr_matrix_1(df_corr)

<seaborn.axisgrid.PairGrid at 0x1a34fd64be0>

This visualization is not suitable due to the great amount of variables. Let's filter out the ones with the highest correlations and visualize those.

df_corr=df_corr.abs()

df_corr=(df_corr.where(~np.tril(np.ones(df_corr.shape)).astype(np.bool)))

df_high_corr=(df_corr[df_corr>0.75].dropna(how="all", axis=0).dropna(how="all", axis=1))

df_high_corr_vars=np.unique(np.concatenate((df_high_corr.columns.values, df_high_corr.index.values)))

df_high_corr = df[df_high_corr_vars]

df_high_corr.shape

(1459, 10)

corr_matrix_1(df_high_corr)

<seaborn.axisgrid.PairGrid at 0x1a303f0ec18>

df_corr.iloc[:,-1].sort_values(ascending=False)[:10]

OverallQual     0.791069
GrLivArea       0.708618
GarageCars      0.640473
GarageArea      0.623423
TotalBsmtSF     0.613905
1stFlrSF        0.605968
FullBath        0.560881
TotRmsAbvGrd    0.533779
YearBuilt       0.523273
YearRemodAdd    0.507430
Name: SalePrice, dtype: float64

The variables which seem to have a high correlation with the target variable are also amongst the ones with the highest correlations in general. Let's also have a look at some of the categorical variables.

Categorical Features¶

CAT_FEATURES =df.select_dtypes(include="object").columns.tolist()

plt.figure(figsize=(40,400))

for i, a in enumerate(CAT_FEATURES):
#     plt.figure(figsize=(5,2))
    plt.subplot(math.ceil(len(CAT_FEATURES)),2,((i+1)*2-1))
    sns.boxplot(x="SalePrice", y=a, data=df)
    plt.ylabel(a, fontsize=40)
    plt.tick_params(axis='y', which='major', labelsize=30)

    plt.subplot(math.ceil(len(CAT_FEATURES)),2,((i+1)*2))
    for b in df[a].unique():
        sns.distplot(df[df[a]==b]["SalePrice"], hist=False, label=a,  kde_kws={"shade": True})
#     plt.ylabel(a, fontsize=40)

#     plt.tight_layout()
# sns.distplot(tsh_sta_pcu.query('Label_trunc == "PCU1"')['Wdf'], hist=False, label='PCU1', kde_kws={"shade": True})
# sns.distplot(tsh_sta_pcu.query('Label_trunc == "PCU2"')['Wdf'], hist=False, label='PCU2', kde_kws={"shade": True});

This gives us a feeling of which attributes might be more important and can help us predict the SalePrice. For these attributes for example the SalePrice is different based on the value:

Alley
Neighborhood
ExterQual
ExterCond
KitchenQual
PoolQual

For BsmtFinType2 e.g. the SalePrice cannot be differentiated much. Hence, this attribute might have less predictive power compared to some of the other attriutes.

The EDA here already shows us potential to predict the SalePrice. Definitively, there are some relationship towards the SalePrice. Next step is to built models to predict the SalePrice and investigate the significance of differences in the attributes.

Explorations based on self-defined categories¶

As mentioned in the beginning, we will also divide the SalePrice in three categories, namely "low", "middle" and "upper class". Thus, we also want to see how these three can be separated and if there is some obvious differences.

sns.distplot(df["SalePrice"], hist=True, label="SalePrice", fit=norm,  kde_kws={"shade": True})

<matplotlib.axes._subplots.AxesSubplot at 0x1a333b33320>

df["SalePrice"].describe()

count      1459.000000
mean     180930.394791
std       79468.964025
min       34900.000000
25%      129950.000000
50%      163000.000000
75%      214000.000000
max      755000.000000
Name: SalePrice, dtype: float64

df["SalePrice"].median()

163000.0

plt.hist(df.SalePrice, bins=10)

(array([148., 722., 373., 135.,  51.,  19.,   4.,   3.,   2.,   2.]),
 array([ 34900., 106910., 178920., 250930., 322940., 394950., 466960.,
        538970., 610980., 682990., 755000.]),
 <a list of 10 Patch objects>)

The range of the SalePrice is from 34,900 - 755,000. The middle class would be the most frequent one. We decided to use the following boundaries:

low: 0-100,000
middle: 100,001-400,000
upper: from 400,001 onwards

# create new column
def price_class(row):
    if row["SalePrice"]<=120000:
        return "low"
    elif (row["SalePrice"]>120000)& (row["SalePrice"]<=250000):
        return "middle"
    else:
        return"upper"


df["Price_Class"]= df.apply(lambda row: price_class(row), axis=1)

plt.figure(figsize=(40,400))

for i, a in enumerate(NUM_FEATURES):
#     plt.figure(figsize=(5,2))
    plt.subplot(math.ceil(len(CAT_FEATURES)),2,((i+1)*2-1))
    sns.boxplot(y=a, x="Price_Class", data=df)
    plt.ylabel(a, fontsize=40)
    plt.tick_params(axis='y', which='major', labelsize=30)

    plt.subplot(math.ceil(len(CAT_FEATURES)),2,((i+1)*2))
    for b in df["Price_Class"].unique():
        sns.distplot(df[df["Price_Class"]==b][a], hist=False, label=b,  kde_kws={"shade": True})
#     plt.ylabel(a, fontsize=40)

#     plt.tight_layout()
# sns.distplot(tsh_sta_pcu.query('Label_trunc == "PCU1"')['Wdf'], hist=False, label='PCU1', kde_kws={"shade": True})
# sns.distplot(tsh_sta_pcu.query('Label_trunc == "PCU2"')['Wdf'], hist=False, label='PCU2', kde_kws={"shade": True});

For some of the numeric features we can already see some very distinguishable boundaries between the classes. E.g. GrdLivArea and GarageArea. These look like promising attributes. Some others like MoSold and BsmtUnfinSF are completely overlapping and will probably not help much in the predictions.

Number of variables	81
Number of observations	1460
Total Missing (%)	5.9%
Total size in memory	924.0 KiB
Average record size in memory	648.1 B

Numeric	38
Categorical	43
Boolean	0
Date	0
Text (Unique)	0
Rejected	0
Unsupported	0

Distinct count	753
Unique (%)	51.6%
Missing (%)	0.0%
Missing (n)	0
Infinite (%)	0.0%
Infinite (n)	0

Mean	1162.6
Minimum	334
Maximum	4692
Zeros (%)	0.0%

Minimum	334
5-th percentile	672.95
Q1	882
Median	1087
Q3	1391.2
95-th percentile	1831.2
Maximum	4692
Range	4358
Interquartile range	509.25

Data exploration of House Price Predictions¶

Table of Contents

Data Profile¶

Overview

Variables

Correlations

Sample